package com.xiaorun.portal.spider.processer;

import com.xiaorun.portal.modal.Vo.News;
import com.xiaorun.portal.modal.Vo.NewsAttach;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Slf4j
public class NewsPageProcesser implements PageProcessor {

    //列表链接
    public static final String URL_LIST = "http://sz.shenkexin.com/project/notices.html";

    //详情页链接
    public static final String URL_DETAIL = "http://sz.shenkexin.com/project/notice-\\d+.html";

    private String regx = "notice-(.*?).html$";

    private Map<String, String> oMap1 = new HashMap<>(16);
    private Map<String, String> oMap2 = new HashMap<>(16);

    //    部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
//    private Site site = Site.me().setDomain("gdsmp.gdyz.org.cn").setRetryTimes(3).setSleepTime(1000).setTimeOut(600000).addHeader("Accept-Encoding", "/").setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36");
    private Site site = Site.me().setDomain("sz.shenkexin.com");

    @Override
    public Site getSite() {
        return site;
    }

    public Date getDateFromStr(String str) {
        String reg = "\\d{4}-\\d{2}-\\d{2}";
        Pattern pattern = Pattern.compile(reg);
        Matcher matcher = pattern.matcher(str);

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        while (matcher.find()) {
            try {
                return sdf.parse(matcher.group());
            } catch (ParseException e) {
                log.error("时间转换出错", e);
            }
        }

        return null;

    }

    @Override
    public void process(Page page) {
        // List<String> links = page.getHtml().links().regex("http://my\\.oschina\\.net/flashsword/blog/\\d+").all();

        if (page.getUrl().regex(URL_LIST).match()) {
            //加入满足条件的链接
            List<String> links = new ArrayList<>();
            links.addAll(page.getHtml().xpath("/html/body/div[10]/div[1]/ul/li/div[1]/h2/a/@href").all());
            page.addTargetRequests(links);

            List<Selectable> selectableList = page.getHtml().xpath("/html/body/div[10]/div[1]/ul/li").nodes();
            if (selectableList != null && selectableList.size() > 0) {
                Pattern p = Pattern.compile(regx);
                for (Selectable selectable : selectableList) {
//                    log.info(selectable.toString());
//                    log.info("+++++++++++++++++++++++++++++++++++++++++");
//
//                    log.info(selectable.xpath("//div/html()").toString().trim());
//                    log.info("__________________________________________");

                    String amount = selectable.xpath("//div[1]/span[4]/text()").toString().trim();
                    String title = selectable.xpath("//h2/a/text()").toString().trim();
                    String url = selectable.xpath("//h2/a/@href").toString();
                    Matcher m = p.matcher(url);
                    while (m.find()) {
                        oMap1.put(m.group(1), amount);
                        oMap2.put(m.group(1), title);
                    }
                }
            }
            page.setSkip(true);
        } else if (page.getUrl().regex(URL_DETAIL).match()) {
            Html html = page.getHtml();

            String tag = html.xpath("/html/body/div[9]/div[1]/h2/a/text()").toString().trim();
            if (StringUtils.isBlank(tag)) {
                page.setSkip(true);
            }

            String summary = html.xpath("/html/body/div[9]/div[1]/p/text()").toString().trim();
            String content = html.xpath("/html/body/div[12]/div[1]/div[2]/div[1]/div/html()").toString().trim();
//            String img;
//            Integer authorId;
//            String status;
//            String applyStatus;

            Date publishDate = getDateFromStr(html.xpath("/html/body/div[9]/div[1]/div/span[1]/text()").toString().trim());

            String projectLevel = html.xpath("/html/body/div[9]/div[1]/div/span[3]/text()").toString().trim();

            String fundEffort = html.xpath("/html/body/div[9]/div[1]/div/span[4]/text()").toString().trim();

            String fundType = html.xpath("/html/body/div[9]/div[1]/div/span[4]/text()").toString().trim();

            String fundMode = html.xpath("/html/body/div[9]/div[1]/div/span[5]/text()").toString().trim();

            String orgDept = html.xpath("/html/body/div[9]/div[2]/ul/li[2]/span/a/text()").toString().trim();

            String dateStr = html.xpath("/html/body/div[9]/div[2]/ul/li[3]/span/text()").toString().trim();
            String[] dateStrArr = dateStr.split("/");
            Date applyStartDate = getDateFromStr(dateStrArr[0]);
            Date applyEndDate = getDateFromStr(dateStrArr[1]);

            String source = page.getUrl().toString();

            String amount = "";
            String title = "";
            Pattern p = Pattern.compile(regx);
            Matcher m = p.matcher(source);
            while (m.find()) {
                String key = m.group(1);
                amount = oMap1.get(key);
                title = oMap2.get(key);
                break;
            }

            News news = new News();
            news.setTag(tag);
            news.setTitle(title);
            news.setSummary(summary);
            news.setContent(content);
            news.setPublishDate(publishDate);
            news.setProjectLevel(projectLevel);
            news.setFundEffort(fundEffort);
            news.setFundType(fundType);
            news.setFundMode(fundMode);
            news.setOrgDept(orgDept);
            news.setApplyStartDate(applyStartDate);
            news.setApplyEndDate(applyEndDate);
            news.setSource(source);
            news.setFundAmount(amount);

            page.putField("news", news);

            List<Selectable> selectableList = page.getHtml().xpath("/html/body/div[12]/div[1]/div[2]/div[2]/div/ul/li").nodes();
            if (selectableList != null && selectableList.size() > 0) {
                List<NewsAttach> newsAttachList = new ArrayList<>();
                for (Selectable selectable : selectableList) {
                    NewsAttach newsAttach = new NewsAttach();
                    String name = selectable.xpath("//a/text()").toString().trim();
                    String link = selectable.xpath("//a/@href").toString().trim();
                    newsAttach.setName(name);
                    newsAttach.setLink("http://sz.shenkexin.com" + link);
                    newsAttachList.add(newsAttach);
                }
                page.putField("newsAttachList", newsAttachList);
            }

        }


    }

    public static void main(String[] args) {
        //http://gdsmp.gdyz.org.cn/sitecn/aspx/productquery.aspx?keyvalue=2017&typename=year
        Spider.create(new NewsPageProcesser()).addUrl("http://sz.shenkexin.com/project/notices.html").addPipeline(new ConsolePipeline()).run();

        //http://sz.shenkexin.com/project/notice-2615.html
        //http://sz.shenkexin.com/project/notice-1.html


    }
}
